<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
  <title>Module: Robotstxt [Robotstxt]</title>
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
  <meta http-equiv="Content-Script-Type" content="text/javascript" />
  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
  <script type="text/javascript">
  // <![CDATA[

  function popupCode( url ) {
    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
  }

  function toggleCode( id ) {
    if ( document.getElementById )
      elem = document.getElementById( id );
    else if ( document.all )
      elem = eval( "document.all." + id );
    else
      return false;

    elemStyle = elem.style;

    if ( elemStyle.display != "block" ) {
      elemStyle.display = "block"
    } else {
      elemStyle.display = "none"
    }

    return true;
  }

  // Make codeblocks hidden by default
  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }<\/style>" )

  // ]]>
  </script>

</head>
<body>


    <div id="classHeader">
        <table class="header-table">
        <tr class="top-aligned-row">
          <td><strong>Module</strong></td>
          <td class="class-name-in-header">Robotstxt</td>
        </tr>
        <tr class="top-aligned-row">
            <td><strong>In:</strong></td>
            <td>


                <a href="../files/lib/robotstxt_rb.html">

                lib/robotstxt.rb

                </a>


        <br />


                <a href="../files/lib/robotstxt/parser_rb.html">

                lib/robotstxt/parser.rb

                </a>


        <br />

            </td>
        </tr>


        </table>
    </div>
  <!-- banner header -->

  <div id="bodyContent">

  <div id="contextContent">

   </div>


    <div id="method-list">
      <h3 class="section-bar">Methods</h3>

      <div class="name-list">

        <a href="#M000001">allowed?</a>&nbsp;&nbsp;

        <a href="#M000002">sitemaps</a>&nbsp;&nbsp;

      </div>
    </div>

  </div>

    <!-- if includes -->

    <div id="section">

    <div id="class-list">
      <h3 class="section-bar">Classes and Modules</h3>

      Class <a href="Robotstxt/Parser.html" class="link">Robotstxt::Parser</a><br />

    </div>

    <div id="constants-list">
      <h3 class="section-bar">Constants</h3>

      <div class="name-list">
        <table summary="Constants">

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">NAME</td>
          <td>=</td>
          <td class="context-item-value">'Robotstxt'</td>

        </tr>

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">GEM</td>
          <td>=</td>
          <td class="context-item-value">'robotstxt'</td>

        </tr>

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">AUTHORS</td>
          <td>=</td>
          <td class="context-item-value">['Simone Rinzivillo &lt;srinzivillo@gmail.com&gt;']</td>

        </tr>

        <tr class="top-aligned-row context-row">
          <td class="context-item-name">VERSION</td>
          <td>=</td>
          <td class="context-item-value">'0.5.2'</td>

        </tr>

        </table>
      </div>
    </div>




    <!-- if method_list -->

    <div id="methods">

      <h3 class="section-bar">Public Class methods</h3>


      <div id="method-M000001" class="method-detail">
        <a name="M000001"></a>

        <div class="method-heading">

          <a href="#M000001" class="method-signature">

          <span class="method-name">allowed?</span><span class="method-args">(url, robot_id)</span>

          </a>

        </div>

        <div class="method-description">

          <p>
Check if the <tt>URL</tt> is allowed to be crawled from the current
<tt>Robot_id</tt>. Robots:Allowed? returns <tt>true</tt> if the robots.txt
file does not block the access to the URL.
</p>
<pre>
 Robotstxt.allowed?('http://www.simonerinzivillo.it/', 'rubytest')
</pre>

          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000001-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000001-source">
<pre>
    <span class="ruby-comment cmt"># File lib/robotstxt.rb, line 35</span>
35:   <span class="ruby-keyword kw">def</span> <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">allowed?</span>(<span class="ruby-identifier">url</span>, <span class="ruby-identifier">robot_id</span>)
36:         
37:           <span class="ruby-identifier">u</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">url</span>)
38:           <span class="ruby-identifier">r</span> = <span class="ruby-constant">Robotstxt</span><span class="ruby-operator">::</span><span class="ruby-constant">Parser</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">robot_id</span>)
39:                 <span class="ruby-identifier">r</span>.<span class="ruby-identifier">allowed?</span>(<span class="ruby-identifier">url</span>) <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">r</span>.<span class="ruby-identifier">get</span>(<span class="ruby-identifier">u</span>.<span class="ruby-identifier">scheme</span> <span class="ruby-operator">+</span> <span class="ruby-value str">'://'</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">u</span>.<span class="ruby-identifier">host</span>)   
40:     
41:   <span class="ruby-keyword kw">end</span>
</pre>
          </div>

        </div>
      </div>


      <div id="method-M000002" class="method-detail">
        <a name="M000002"></a>

        <div class="method-heading">

          <a href="#M000002" class="method-signature">

          <span class="method-name">sitemaps</span><span class="method-args">(url, robot_id)</span>

          </a>

        </div>

        <div class="method-description">

          <p>
Analyze the robots.txt file to return an <tt>Array</tt> containing the list
of XML Sitemaps URLs.
</p>
<pre>
 Robotstxt.sitemaps('http://www.simonerinzivillo.it/', 'rubytest')
</pre>

          <p><a class="source-toggle" href="#"
            onclick="toggleCode('M000002-source');return false;">[Source]</a></p>
          <div class="method-source-code" id="M000002-source">
<pre>
    <span class="ruby-comment cmt"># File lib/robotstxt.rb, line 47</span>
47:   <span class="ruby-keyword kw">def</span> <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">sitemaps</span>(<span class="ruby-identifier">url</span>, <span class="ruby-identifier">robot_id</span>)
48:         
49:           <span class="ruby-identifier">u</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">url</span>)
50:           <span class="ruby-identifier">r</span> = <span class="ruby-constant">Robotstxt</span><span class="ruby-operator">::</span><span class="ruby-constant">Parser</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">robot_id</span>)
51:                 <span class="ruby-identifier">r</span>.<span class="ruby-identifier">sitemaps</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">r</span>.<span class="ruby-identifier">get</span>(<span class="ruby-identifier">u</span>.<span class="ruby-identifier">scheme</span> <span class="ruby-operator">+</span> <span class="ruby-value str">'://'</span> <span class="ruby-operator">+</span> <span class="ruby-identifier">u</span>.<span class="ruby-identifier">host</span>)        
52:     
53:   <span class="ruby-keyword kw">end</span>
</pre>
          </div>

        </div>
      </div>



    </div>




  </div>

<div id="validator-badges">
  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
</div>

</body>
</html>
